{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1e664852-ac48-43a9-9457-6827414702cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import logging\n",
    "import os\n",
    "import subprocess\n",
    "import sys\n",
    "import time\n",
    "from os.path import dirname\n",
    "\n",
    "from drain3 import TemplateMiner\n",
    "from drain3.template_miner_config import TemplateMinerConfig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6553a5fd-76c7-4995-a0ae-59781a786f56",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def hash_decimal_to_hex(decimal):\n",
    "    # 使用Python内置哈希函数将整数哈希为一个大整数\n",
    "    hashed_decimal = abs(hash(str(decimal)))\n",
    "    # 将大整数转换为16进制字符串\n",
    "    hex_string = hex(hashed_decimal)\n",
    "    # 取字符串末尾8个字符作为哈希值，即一个长度为8的16进制数\n",
    "    hash_value = hex_string[:8]\n",
    "    # 将16进制数转换为整数并返回\n",
    "    return hash_value\n",
    "    \n",
    "def get_drain_template(log_df):\n",
    "    logger = logging.getLogger(__name__)\n",
    "    logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(message)s')\n",
    "\n",
    "    config = TemplateMinerConfig()\n",
    "    config.load(\"drain3.ini\")\n",
    "    config.profiling_enabled = True\n",
    "    template_miner = TemplateMiner(config=config)\n",
    "\n",
    "    line_count = 0\n",
    "\n",
    "    lines = log_df['message'].tolist()\n",
    "\n",
    "    start_time = time.time()\n",
    "    batch_start_time = start_time\n",
    "    batch_size = 10000\n",
    "    result_json_list = []\n",
    "    for line in lines:\n",
    "        line = line.rstrip()\n",
    "        line = line.split(',',5)[-1]\n",
    "        result = template_miner.add_log_message(line)\n",
    "        line_count += 1\n",
    "        if line_count % batch_size == 0:\n",
    "            time_took = time.time() - batch_start_time\n",
    "            rate = batch_size / time_took\n",
    "            batch_start_time = time.time()\n",
    "        if result[\"change_type\"] != \"none\":\n",
    "            result_json = json.dumps(result)\n",
    "        result_json_list.append(result)\n",
    "\n",
    "    time_took = time.time() - start_time\n",
    "    rate = line_count / time_took\n",
    "\n",
    "    sorted_clusters = sorted(template_miner.drain.clusters, key=lambda it: it.size, reverse=True)\n",
    "    for cluster in sorted_clusters:\n",
    "        logger.info(cluster)\n",
    "\n",
    "    # with open('logstash_structured.json', 'w+') as f:\n",
    "    #     json.dump(result_json_list, f)\n",
    "\n",
    "    # with open('logstash_templates.txt', 'w+') as f:\n",
    "    #     for cluster in sorted_clusters:\n",
    "    #         f.write(str(cluster))\n",
    "    #         f.write('\\n')\n",
    "\n",
    "    EID_list = []        \n",
    "    for logdict in result_json_list:\n",
    "        EID_list.append(hash_decimal_to_hex(logdict['cluster_id']))\n",
    "\n",
    "    log_df['template_id'] = EID_list\n",
    "\n",
    "    return log_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "acc53da3-3fe9-41ad-81fd-bc68bcf68fbe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "        if (window._pyforest_update_imports_cell) { window._pyforest_update_imports_cell('import pandas as pd'); }\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "config file not found: drain3.ini\n",
      "Starting Drain3 template miner\n",
      "ID=5     : size=17076     : message:conversion request successful\n",
      "ID=1     : size=16086     : message:request <*>\n",
      "ID=3     : size=6378      : message:Getting supported currencies...\n",
      "ID=4     : size=6378      : GetCartAsync called with <*>\n",
      "ID=7     : size=4505      : message:failed to retrieve ads\n",
      "ID=19    : size=3746      : message:[GetQuote] <*> request\n",
      "ID=2     : size=3645      : message:serving product page\n",
      "ID=21    : size=1612      : message:view user cart\n",
      "ID=9     : size=860       : message:home\n",
      "ID=10    : size=860       : message:ENV_PLATFORM is: local\n",
      "ID=20    : size=790       : message:adding to cart\n",
      "ID=6     : size=647       : '2ZYFJ3GM2N']\n",
      "ID=13    : size=625       : 'LS4PSXUNUM']\n",
      "ID=11    : size=624       : '0PUK6V6EV0']\n",
      "ID=18    : size=620       : '1YMWWN1N4O']\n",
      "ID=15    : size=616       : '9SIQT8TOJO']\n",
      "ID=40    : size=607       : '66VCHSJNUP']\n",
      "ID=24    : size=593       : '6E92ZMYYFZ']\n",
      "ID=33    : size=593       : 'OLJCESPC7Z']\n",
      "ID=12    : size=589       : 'L9ECAV7KIM']\n",
      "ID=8     : size=575       : message:setting currency\n",
      "ID=16    : size=550       : <*> metricsets\n",
      "ID=25    : size=522       : message:[ShipOrder] <*> request\n",
      "ID=14    : size=300       : Checking CartService Health\n",
      "ID=22    : size=261       : message:placing order\n",
      "ID=26    : size=261       : EmptyCartAsync called with <*>\n",
      "ID=27    : size=261       : \"credit_card_cvv\":672,\"credit_card_expiration_year\":2039,\"credit_card_expiration_month\":1}}\n",
      "ID=28    : size=261       : message:Transaction processed: visa ending 0454 Amount: <*>\n",
      "ID=29    : size=261       : message:payment went through (transaction_id: <*>\n",
      "ID=30    : size=261       : message:order placed\n",
      "ID=31    : size=261       : message:A request to send order confirmation email to someone@example.com has been received.\n",
      "ID=32    : size=261       : message:order confirmation email sent to \"someone@example.com\"\n",
      "ID=43    : size=200       : message:gathering metrics\n",
      "ID=35    : size=161       : quantity=1\n",
      "ID=36    : size=134       : quantity=3\n",
      "ID=38    : size=134       : quantity=5\n",
      "ID=34    : size=131       : quantity=10\n",
      "ID=17    : size=119       : quantity=2\n",
      "ID=37    : size=111       : quantity=4\n",
      "ID=45    : size=67        : message:[PlaceOrder] <*> user_currency=\"CAD\"\n",
      "ID=23    : size=66        : message:[PlaceOrder] <*> user_currency=\"EUR\"\n",
      "ID=44    : size=63        : message:[PlaceOrder] <*> user_currency=\"JPY\"\n",
      "ID=42    : size=51        : message:[PlaceOrder] <*> user_currency=\"USD\"\n",
      "ID=41    : size=14        : message:[PlaceOrder] user_id=\"185507d3-b509-4d66-b4ba-13658d032e73\" <*>\n",
      "ID=39    : size=4         : message:[Recv ListRecommendations] product_ids=['LS4PSXUNUM']\n"
     ]
    }
   ],
   "source": [
    "base_path = '../origin_data/demo/log/'\n",
    "log_df = pd.read_csv(base_path+'log_logstash-service.csv')\n",
    "template_df = get_drain_template(log_df)\n",
    "template_df.to_csv(base_path+'template_logstash-service.csv', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8f5125cc-9db0-4092-adfc-8729c7e77197",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
